import unittest
import time
import re
from urllib import request
from urllib import error
from lxml import etree
from fontTools.ttLib import TTFont
from io import BytesIO
import requests
import pymysql

class mysql:
    db=pymysql.connect
    def __init__(self,url,username,pwd,dbname):
        self.db = pymysql.connect(url,username,pwd,dbname)
    def __del__(self):
        self.db.close()
    def do(self,sql):
        try:
            cursor = self.db.cursor()
            cursor.execute(sql)
            self.db.commit()
            return cursor.fetchall()
        except Exception:
            print(Exception)

def get_font(url):
    time.sleep(1)
    response = requests.get(url)
    font = TTFont(BytesIO(response.content))
    cmap = font.getBestCmap()  # 返回unicode cmap字典可用的字体
    font.close()
    return cmap

header = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
    ,
    'cookie': '_qda_uuid=33835f41-d355-ab63-b6b1-b181fade7631; _yep_uuid=5370cc30-ab86-0c1b-a9ed-1bf80716979e; e2=%7B%22pid%22%3A%22qd_P_xiangqing%22%2C%22eid%22%3A%22%22%7D; e1=%7B%22pid%22%3A%22qd_P_mulu%22%2C%22eid%22%3A%22qd_G15%22%2C%22l1%22%3A3%7D; _csrfToken=LNl6uV6HsfNXhAtQCBv4GEWN0re4iAjBIeMmwBS6; newstatisticUUID=1592315328_1232023422; qdrs=0%7C3%7C0%7C0%7C1; showSectionCommentGuide=1; qdgd=1; hiijack=0; rcr=1023181440%2C1024102897%2C1023872747%2C1%2C1023820549%2C1019605599%2C1023683481%2C1021700509%2C1023468159%2C1023434582%2C1023516475%2C1022976131%2C1023621065%2C1019896310%2C1022080664%2C1023380618%2C1021701291%2C1020764004%2C1022939709%2C1022568233%2C1022052397%2C1022635372%2C1022996412%2C1022206306%2C1022297827%2C1022330425%2C1022854245%2C1022617589%2C1017814973%2C1023074106%2C1022384177%2C1022326509%2C1022644809%2C1021160568%2C1021697174%2C1021375629%2C1021648770%2C1020422944; lrbc=1023181440%7C567128757%7C1%2C1024102897%7C580311797%7C0%2C1020422944%7C529845795%7C0; _yep_uuid=3bf3cd4d-bf4f-2bde-f12e-87e4829c5ded; e1=%7B%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A103%22%2C%22l1%22%3A8%7D; e2=%7B%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A72%22%2C%22l1%22%3A3%7D'
    , 'upgrade-insecure-requests': '1'
    }

url = 'https://book.qidian.com/info/'

def payige(bookId):
    try:
        # 包装请求
        req = request.Request(url + str(bookId), headers=header)
        # 发送请求
        html = request.urlopen(req).read().decode('utf-8')
    except error.HTTPError :
        return {}
    else :
        #用etree解析HTML
        html = etree.HTML(html)
        #解密字体
        bookPasswordUrl = html.xpath('/html/body/div/div[6]/div[1]/div[2]/p[3]/em[1]/style/text()')  # 起点文字库的三个URL
        trueUrl = re.search('woff.*?url.*?\'(.+?)\'.*?truetype', bookPasswordUrl[0]).group(1)  # 正则提取truetype也就是起点字典的URL，这个URL是随时变动的，所以只能查一次找一次
        encode = get_font(trueUrl)  # 访问此URL，将文件存在缓存区，再解析成字典
        WORD_MAP = {'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5', 'six': '6','seven': '7','eight': '8', 'nine': '9', 'period': '.'}

        #书名
        bookName = html.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/em/text()')[0]

        #tag
        bookTagArr = html.xpath('/html/body/div/div[6]/div[1]/div[2]/p[1]/a/text()')
        bookType = bookTagArr[0]
        tagNum = len(bookTagArr)
        bookTag = ''
        tagNo = 0
        for i in bookTagArr:
            tagNo+=1
            if(tagNo == 1):
                continue
            if(tagNo!=tagNum):
                bookTag += (bookTagArr[tagNo-1]+'|')
            if(tagNo == tagNum):
                bookTag += bookTagArr[tagNo-1]

        #作者
        zuozhe = html.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/span/a/text()')[0]

        #发布时间
        time = html.xpath('//*[@id="j-catalogWrap"]/div[2]/div/ul/li/a[1]/@title')
        if(len(time)>=2):
            startTime = re.findall('首发时间：(.+?) ',time[0])[0]
            endTime = re.findall('首发时间：(.+?) ',time[-1])[0]
        else:
            startTime = re.findall('首发时间：(.+?) ',time[0])[0]
            endTime = re.findall('首发时间：(.+?) ',time[0])[0]

        #字数
        bookZishu = html.xpath(
            '/html/body/div/div[6]/div[1]/div[2]/p[3]/em[1]/span/text()')  # 爬取需要的信息 这个是字数，不过起点做了反爬虫，他有自己的文字库
        haveWan = html.xpath('/html/body/div/div[6]/div[1]/div[2]/p[3]/cite[1]/text()')[0] == '万字'
        realZiShu = ''
        for i in bookZishu[0]:
            realZiShu += WORD_MAP[encode[ord(i)]]
        if haveWan:
            realZiShu = int(float(realZiShu)*10000)
        else:
            realZiShu = int(realZiShu)

        #推荐
        bookTuiJian = html.xpath(
            '/html/body/div/div[6]/div[1]/div[2]/p[3]/em[2]/span/text()')  # 推荐
        haveWan = html.xpath('/html/body/div/div[6]/div[1]/div[2]/p[3]/cite[2]/text()')[0] == '万总推荐'
        realTuiJian = ''
        for i in bookTuiJian[0]:
            realTuiJian += WORD_MAP[encode[ord(i)]]
        if haveWan:
            realTuiJian = int(float(realTuiJian)*10000)
        else:
            realTuiJian = int(realTuiJian)

        return {'bookId':bookId,
                'name': bookName,
                'zishu':realZiShu,
                'tuijian':realTuiJian,
                'type':bookType,
                'tag':bookTag,
                'startTime':startTime,
                'endTime':endTime,
                'zuozhe':zuozhe,
                }


def paduoge():
    a = mysql("47.94.92.138", "cq", "123456", "novel")
    #上次停的地方,从 1020000000 开始的
    shangcitingdedifang = 1020001522
    bookId = 1020001522
    while bookId < 1031116068:
        yibenshu = payige(bookId)
        if(yibenshu != {}):
            saveData(yibenshu,a)
            time.sleep(0.7)
        print(yibenshu)
        bookId += 1
        time.sleep(0.1)

def saveData(onebook,db):
    print(onebook)
    db.do('INSERT INTO novel'
          '(novel.`name`,'
          'website,'
          'type,'
          'tag,'
          'startTime,'
          'endTime,'
          'zuozhe,'
          'zishu,'
          'yuanBookId,'
          'tuijian) '
          'VALUES("{}","起点","{}","{}","{}","{}","{}","{}","{}","{}")'.
          format(onebook['name'],
                 onebook['type'],
                 onebook['tag'],
                 onebook['startTime'],
                 onebook['endTime'],
                 onebook['zuozhe'],
                 onebook['zishu'],
                 onebook['bookId'],
                 onebook['tuijian'])
          )
    pass

if __name__ == '__main__':
    paduoge()

